import sys
from os import path, walk
from datetime import datetime, timedelta
from codecs import open

__dir__ = path.dirname(path.abspath(__file__))
path_error = path.join(__dir__, "etllog.err")
sys.stderr = open(path_error, 'a')

report='E:\Py\MyPy\etllog\scripts\etl.log'
scripts='E:\Py\MyPy\etllog\scripts'
exceptions=['archive', 'logs', 'manual']
keywords=['  INFO : Finished','  WARN : ',' ERROR : ']

def findlogs(dir, excl):
	"Function returns all items in dir excluding ones that are in dirs listed in excl"
	top=path.abspath(dir)
	logs=[]
	tuples=walk(top)
	next(tuples)
	for tuple in tuples:
		counter=0
		for name in tuple[0].lower().split('\\'):
			if name in excl:
				counter+=1
		if counter==0:
			for file in tuple[2]:
				if file[-3:]=='log':
					logs+=[path.abspath(tuple[0]+'\\'+file)]
		
	return logs

def line_to_dt(line):
	result=[]
	if line[:4].isdigit():
		result+=[int(line[:4])]
	else:
		result+=[]
	if line[5:7].isdigit():
		result+=[int(line[5:7])]
	else:
		result+=[]
	if line[8:10].isdigit():
		result+=[int(line[8:10])]
	else:
		result+=[]
	if line[11:13].isdigit():
		result+=[int(line[11:13])]
	else:
		result+=[]
	return result

def validdate(line):
	now=datetime.now()
	delta=timedelta(days=-1)
	dt_valid=[[now.year, (now+delta).year], 
	[now.month, (now+delta).month], 
	[now.day, (now+delta).day], 
	[11, 18]]
	dt_line=line_to_dt(line)
	result=False
	if dt_line!=[] and ((dt_line[0]==dt_valid[0][0] 
		and dt_line[1]==dt_valid[1][0]
		and dt_line[2]==dt_valid[2][0]
		and dt_line[3]<dt_valid[3][0])
		or (dt_line[0]==dt_valid[0][1] and
			dt_line[1]==dt_valid[1][1] and
			dt_line[2]==dt_valid[2][1] and
			dt_line[3]>dt_valid[3][1])):
		result=True
	return result

def validkey(line, keys):
	result=False
	for key in keys:
		if key in line:
			result=True
			break
	return result

def append(log, end, keys):
	logfile = open(path.abspath(log), 'r', 'cp1251').read().splitlines()
	endfile = open(path.abspath(end), 'a', 'utf-8')
	endfile.write(log+'\n')
	endfile.write('\n')
	count=0
	for line in logfile:
		if validdate(line) and validkey(line, keys):
			endfile.write(line+'\n')
			count+=1
	endfile.write('\n')
	endfile.close()
	return [len(logfile), count]
			
			
def collect(end=path.abspath(report), dir=path.abspath(scripts), excl=exceptions, keys=keywords):
	logs=findlogs(dir, excl)
	endfile = open(path.abspath(end), 'a', 'utf-8')
	endfile.write('@=~=(=+=)=~=@'*7)
	endfile.write('\n\n\n')
	endfile.write('ETL logs for: '+str(datetime.now())[:10])
	endfile.write('\n\n')
	endfile.close()
	count=[0,0]
	for log in logs:
		result=append(log, end, keys)
		count[0]+=result[0]
		count[1]+=result[1]
	return count


collect()

sys.stderr.close()
sys.stderr = sys.__stderr__

import shortlog